from v5.Spider import Spider
from v5.SpiderModule import SpiderModule
from v5.test.TestPipeline import TestPipeline
from v5.test.TestPageProcessor import TestProcessor
from v5.SockPuppet import SockPuppet
from v5.RequestItem import RequestItem
from v5.test.headers_list import headers_list


if __name__ == '__main__':
    headers_list = [
        {
            'cookie': '__cfduid=d1437ed867d4e0d49498d3ae60ef4d2221619946145; MAID=RfCulBxlwcBtHd6ScOA3jQ==; I2KBRCK=1; _ga=GA1.2.1292260820.1619946150; _hjid=d2979a3e-6e47-4657-9be2-8fe40303bc37; Pastease.passive.activated.5YhMrk04JDZQkJe=0; Pastease.passive.chance.5YhMrk04JDZQkJe=chance73.3; cookiePolicy=accept; JSESSIONID=769f9d92-fc26-4663-8dac-1e8a9e9037d8; SERVER=WZ6myaEXBLFM4S5S3dlZcA==; MACHINE_LAST_SEEN=2021-05-08T01:17:10.908-07:00; _gid=GA1.2.1205337192.1620461833; _hjIncludedInPageviewSample=1; _hjTLDTest=1; _hjAbsoluteSessionInProgress=0; _hjIncludedInSessionSample=1; _hp2_ses_props.1083010732={"ts":1620461833693,"d":"dl.acm.org","h":"/doi/proceedings/10.1145/3318464"}; __atuvc=18|18; __atuvs=60964909d7856f9f002; _hp2_id.1083010732={"userId":"7867299362336533","pageviewId":"5523636372749531","sessionId":"1631770374156406","identity":null,"trackerVersion":"4.0"}',
            'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36'
        }
    ]
    sock_puppet = SockPuppet()
    sock_puppet.set_list(headers_list)
    spider_module = SpiderModule(sock_puppet=sock_puppet,
                                 pipeline_class=TestPipeline,
                                 page_processor_class=TestProcessor)
    request_item = RequestItem(url='https://dl.acm.org/doi/proceedings/10.1145/3325480?tocHeading=heading14')
    spider = Spider(spider_module_list=[spider_module], thread_num=1, request_items=[request_item])
    spider.run()